﻿using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Net;
using System.Net.Mail;
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Web;
using System.Xml;
using IEx.Common.Sources;
using IEx.Model.Partials;
using IEx.Model.Partials.Media;
using iTextSharp.text.pdf;
using iTextSharp.text.pdf.parser;
using IEx.Common;

namespace IEx.Utilities
{
    public class Utility
    {
        #region Fields

        public static WebRequestRetryPolicy retry = new WebRequestRetryPolicy();
        public static object lockFailRequest = new object();
        public static object lockInsertNews = new object();
        public static object lockInsertForums = new object();
        private const string FailRequestsFileName = "FailRequests.xml";
        private const string FailInsertingNewsFileName = "FailInsertingNews.xml";
        private const string IgnoreAnnouncementFileName = "IgnoreAnnouncement.xml";
        private const string ForumAnnouncementFileName = "ForumAnnouncement.xml";

        private static TimeZoneInfo CETTimezoneInfo = TimeZoneInfo.FindSystemTimeZoneById("Central Europe Standard Time");
        private static TimeZoneInfo GMTTimezoneInfo = TimeZoneInfo.FindSystemTimeZoneById("GMT Standard Time");
        static IOStreamReliable rely = new IOStreamReliable(4, 2000);
        private static object lockObject = new Object();

        private const int CorrectSignatureLength = 81;
        private const string SignatureQuery = "signature";

        #endregion

        /// <summary>
        /// 
        /// </summary>
        /// <param name="toEmailAddress"></param>
        /// <param name="subject"></param>
        /// <param name="emailContent"></param>
        public static void SendEmail(string toEmailAddress, string subject, string emailContent)
        {
            DataFeedConfiguration Setting = DataFeedConfiguration.GetConfig();
            // we are using System.Net.Mail.SmtpClient in 2.0
            int smtpPort = 0;

            int.TryParse(Setting.EmailSetting.SmtpPort, out smtpPort);
            SmtpClient smtpClient = new SmtpClient(Setting.EmailSetting.SmtpHost, (smtpPort == 0 ? 25 : smtpPort));

            smtpClient.Credentials = new System.Net.NetworkCredential(Setting.EmailSetting.SenderEmailAddress, Setting.EmailSetting.SenderEmailPassword);
            MailMessage message = new MailMessage();

            message.From = new MailAddress(Setting.EmailSetting.SenderEmailAddress, Setting.EmailSetting.SenderAlias);
            // To address collection of MailAddress                 
            string[] arrReceiver = null;
            if (toEmailAddress.IndexOf(';') != -1)
            {
                arrReceiver = Setting.EmailSetting.ToEmailAddress.Split(';');
            }
            if (arrReceiver == null)
            {
                if (RegexUtility.ValidateEmail(toEmailAddress.Trim()))
                    message.To.Add(new MailAddress(toEmailAddress.Trim()));
            }
            else
            {
                foreach (string to in arrReceiver)
                {
                    if (RegexUtility.ValidateEmail(to.Trim()))
                        message.To.Add(new MailAddress(to.Trim()));
                }
            }
            //if (ValidateEmail(Setting.EmailSetting.CcEmailAddress))
            //    message.CC.Add(new MailAddress(Setting.EmailSetting.CcEmailAddress));
            //email subject
            message.Subject = subject;
            message.IsBodyHtml = true;
            // Message body content
            message.Body = emailContent;
            // Send SMTP mail
            smtpClient.Send(message);
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="extention"></param>
        /// <returns></returns>
        public static string GetMIME(string extention)
        {
            string contentType = string.Empty;
            switch (extention.ToLower())
            {
                case ".pdf":
                    contentType = "application/pdf";
                    break;
                case ".rtf":
                    contentType = "application/rtf";
                    break;
                case ".xls":
                    contentType = "application/vnd.ms-excel";
                    break;
                case ".xml":
                    contentType = "text/xml";
                    break;
                case ".html":
                case ".htm":
                    contentType = "text/html";
                    break;
                case ".bmp":
                    contentType = "image/bmp";
                    break;
                case ".gif":
                    contentType = "image/gif";
                    break;
                case ".jpeg":
                case ".jpg":
                case ".jpe":
                case ".jfif":
                case ".pjpeg":
                case ".pjp":
                    contentType = "image/jpeg";
                    break;
                case ".png":
                    contentType = "image/png";
                    break;
                case ".tiff":
                case ".tif":
                    contentType = "image/tiff";
                    break;
                case ".mp4":
                    contentType = "video/mp4";
                    break;
                case ".m4v":
                    contentType = "video/m4v";
                    break;
                case ".ogg":
                    contentType = "video/ogg";
                    break;
                case ".ogv":
                    contentType = "video/ogv";
                    break;
                case ".webm":
                    contentType = "video/webm";
                    break;
                case ".mov":
                    contentType = "video/mov";
                    break;
                case ".mpeg4":
                    contentType = "video/mpeg4";
                    break;
                case ".avi":
                    contentType = "video/avi";
                    break;
                case ".wmv":
                    contentType = "video/wmv";
                    break;
                case ".mpegps":
                    contentType = "video/mpegps";
                    break;
                case ".flv":
                    contentType = "video/flv";
                    break;
                case ".3gpp":
                    contentType = "video/3gpp";
                    break;
                default:
                    break;
            }
            return contentType;
        }

        /// <summary>
        /// 
        /// </summary>
        public static string DownloadedFolderPath
        {
            get
            {
                //Make full pdf folder path by combine
                return DataFeedConfiguration.GetConfig().DownloadedFolder.Path;
            }
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="sourceId"></param>
        /// <returns></returns>
        public static string GetSourceCodeFolderName(int sourceId)
        {
            string folderName = sourceId.ToString();
            switch (sourceId)
            {
                case (int)SourceIds.Facebook:
                    folderName = "FaceBook";
                    break;
                case (int)SourceIds.Twitter:
                    folderName = "Twitter";
                    break;
                case (int)SourceIds.Youtube:
                    folderName = "Youtube";
                    break;
                case (int)SourceIds.GooglePlus:
                    folderName = "GooglePlus";
                    break;
                case (int)SourceIds.Baomoi:
                    folderName = "Baomoi";
                    break;
                case (int)SourceIds.Lamchame:
                    folderName = "Lamchame";
                    break;
                case (int)SourceIds.Otofun:
                    folderName = "Otofun";
                    break;
                case (int)SourceIds.Rongbay:
                    folderName = "Rongbay";
                    break;
                case (int)SourceIds.Vatgia:
                    folderName = "Vatgia";
                    break;
                case (int)SourceIds.Webtretho:
                    folderName = "Webtretho";
                    break;
                case (int)SourceIds.Tinhte:
                    folderName = "Tinhte";
                    break;
                case (int)SourceIds.Tamtay:
                    folderName = "Tamtay";
                    break;
                case (int)SourceIds.Clipvn:
                    folderName = "Clipvn";
                    break;
                case (int)SourceIds.HKExNews:
                    folderName = "HKExNews";
                    break;
                case (int)SourceIds.Foody:
                    folderName = "Foody";
                    break;
                case (int)SourceIds.MonngonMoingay:
                    folderName = "MonngonMoingay";
                    break;
                case (int)SourceIds.Amthuc365:
                    folderName = "Amthuc365";
                    break;
                case (int)SourceIds.NhahangHanoi:
                    folderName = "NhahangHanoi";
                    break;
                case (int)SourceIds.Chotot:
                    folderName = "Chotot";
                    break;
                case (int)SourceIds.Raovat:
                    folderName = "Raovat";
                    break;
            }
            return folderName;
        }

        /// <summary>
        /// 
        /// </summary>
        /// <returns></returns>
        public static string GetSignatureQuery()
        {
            return SignatureQuery;
        }

        #region Methods to extract pdf's content

        /// <summary>
        /// Read content of pdf file
        /// </summary>
        /// <param name="filePath"></param>
        /// <returns></returns>
        public static string ReadPdfFile(string filePath)
        {
            if (IsFileValid(filePath))
            {
                try
                {
                    StringBuilder text = new StringBuilder();
                    //FileStream fStream = new FileStream(filePath, FileMode.Open, FileAccess.Read);
                    //PdfReader pdfReader = new PdfReader(fStream);

                    // 
                    // Using Stream to read pdf file's content sometimes raised error 
                    // "The process cannot access the file ... because it is being used by another process" because iTextSharp
                    // is using Stream to read content.
                    // Force the iTextSharp loads all the content into memory. This solution may cause the error "Out of memory"
                    // if the program reads too many files.
                    // TODO: try to copy file to a temporary location then read from that, after finish reading, just delete temp file
                    PdfReader pdfReader = new PdfReader(File.ReadAllBytes(filePath));
                    try
                    {
                        if (pdfReader.IsOpenedWithFullPermissions)
                        {
                            for (int page = 1; page <= pdfReader.NumberOfPages; page++)
                            {
                                ITextExtractionStrategy strategy = new SimpleTextExtractionStrategy();
                                string currentText = PdfTextExtractor.GetTextFromPage(pdfReader, page, strategy);

                                text.Append(currentText);
                            }
                        }
                    }
                    finally
                    {
                        if (pdfReader != null)
                        {
                            //pdfReader.Close();
                            pdfReader.Dispose();
                        }
                        //fStream.Close();
                    }
                    return text.ToString();
                }
                catch (Exception ex)
                {
                    Logger.WriteError(string.Format("Has an itextsharp error, cannot read content from pdf file: {0}. Insert blank string into DB!", filePath), ex);
                }
            }
            else
            {
                Logger.WriteError(string.Format("Cannot open the file or file is not available: {0}: ", filePath), null);
            }

            return string.Empty;
        }

        #endregion

        /// <summary>
        /// Check for a file whether exist and can be readable
        /// </summary>
        /// <param name="filePath"></param>
        /// <returns></returns>
        private static bool IsFileValid(string filePath)
        {
            if (!File.Exists(filePath))
                return false;
            try
            {
                File.Open(filePath, FileMode.Open, FileAccess.Read).Dispose();
                return true;
            }
            catch (IOException)
            {
                return false;
            }
        }

        /// <summary>
        /// Get Html File Attachment
        /// </summary>
        /// <param name="htmLike"></param>
        /// <returns></returns>
        public static string LoadHtml(string url, ref HttpStatusCode status)
        {
            try
            {
                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
                request.UserAgent = "Foo";
                request.Accept = "*/*";
                string html = "";
                // get response html
                retry.ExecuteAction(() =>
                {
                    HttpWebResponse response = request.GetResponse() as HttpWebResponse;
                    string charset = response.CharacterSet;

                    MemoryStream memoryStream = new MemoryStream();
                    using (Stream responseStream = response.GetResponseStream())
                    {

                        byte[] buffers = new byte[1024];
                        int byteCount = 0;
                        do
                        {
                            byteCount = responseStream.Read(buffers, 0, buffers.Length);
                            memoryStream.Write(buffers, 0, byteCount);
                        } while (byteCount > 0);
                    }

                    memoryStream.Seek(0, SeekOrigin.Begin);
                    StreamReader str = new StreamReader(memoryStream, Encoding.GetEncoding(charset) ?? Encoding.Unicode);
                    string htmlContent = str.ReadToEnd();
                    // Get realcharset from html
                    string realCharset = GetCharSetInMetaTag(htmlContent);

                    if (!string.IsNullOrEmpty(realCharset) && !realCharset.Equals(charset, StringComparison.OrdinalIgnoreCase))
                    {
                        memoryStream.Seek(0, SeekOrigin.Begin);
                        str = new StreamReader(memoryStream, Encoding.GetEncoding(realCharset));
                        htmlContent = str.ReadToEnd();
                    }

                    html = htmlContent;
                });
                return html;
            }
            catch (WebException ex)
            {
                if (ex.Status == WebExceptionStatus.Timeout)
                {
                    status = HttpStatusCode.RequestTimeout;
                    return string.Empty;
                }

                if (ex.Status == WebExceptionStatus.ProtocolError)
                {
                    status = HttpStatusCode.NotFound;
                }
                return string.Empty;
            }

        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="html"></param>
        /// <returns></returns>
        public static string GetCharSetInMetaTag(string html)
        {
            try
            {
                Regex regexMetaTag = new Regex("(?<=<meta(.*)charset=)([^\"'>]*)", RegexOptions.IgnoreCase);
                string chartSetValue = string.Empty;
                Match matchResults = regexMetaTag.Match(html);
                while (matchResults.Success)
                {
                    for (int i = 1; i < matchResults.Groups.Count; i++)
                    {
                        Group groupObj = matchResults.Groups[i];
                        if (groupObj.Success)
                        {
                            chartSetValue = groupObj.Value.Trim();
                        }
                    }
                    matchResults = matchResults.NextMatch();
                }
                //In Case Clip.vn charset has style special
                if (string.IsNullOrEmpty(chartSetValue))
                {
                    regexMetaTag = new Regex("(?<=<meta(.*)charset=\")([^\"'>]*)", RegexOptions.IgnoreCase);
                    chartSetValue = string.Empty;
                    matchResults = regexMetaTag.Match(html);
                    while (matchResults.Success)
                    {
                        for (int i = 1; i < matchResults.Groups.Count; i++)
                        {
                            Group groupObj = matchResults.Groups[i];
                            if (groupObj.Success)
                            {
                                chartSetValue = groupObj.Value.Trim();
                            }
                        }
                        matchResults = matchResults.NextMatch();
                    }
                }
                return chartSetValue;
            }
            catch
            {
                return string.Empty;
            }
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="link"></param>
        /// <returns></returns>
        public static string GetFilename(string link)
        {
            int index = link.TrimEnd('/').LastIndexOf('/');
            if (index > 0)
            {
                return link.Substring(index + 1);
            }
            else
            {
                return string.Empty;
            }
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="link"></param>
        /// <returns></returns>
        public static string GetFileExtention(string link)
        {
            int index = link.LastIndexOf('.');
            if (index > 0)
            {
                return link.Substring(index);
            }
            else
                return string.Empty;
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="path"></param>
        /// <returns></returns>
        public static string RemoveIllegalPathCharacters(string path)
        {
            string regexSearch = new string(Path.GetInvalidFileNameChars()) + new string(Path.GetInvalidPathChars());
            var r = new Regex(string.Format("[{0}]", Regex.Escape(regexSearch)));
            return r.Replace(path, "");
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="attachmentPath"></param>
        /// <param name="sourceId"></param>
        /// <param name="defaultOrder"></param>
        /// <returns></returns>
        public static int GetAttachmentOrder(string attachmentPath, SourceIds sourceId, int defaultOrder = 0)
        {
            int order = 0;
            switch (sourceId)
            {
                case SourceIds.Amthuc365:
                    order = defaultOrder;
                    break;
                case SourceIds.Baomoi:
                    order = 0;
                    break;
                case SourceIds.Clipvn:
                    order = 0;
                    break;
                case SourceIds.HKExNews:
                    order = 0;
                    break;
            }

            return order;
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="extention"></param>
        /// <returns></returns>
        public static int GetAttachmentOrderByExtention(string extention)
        {
            int order = 0;
            switch (extention.ToLower())
            {
                case ".rtf":
                    order = 0;
                    break;
                case ".pdf":
                    order = 1;
                    break;
                case ".xls":
                    order = 2;
                    break;
                case ".xml":
                    order = 3;
                    break;
            }

            return order;
        }

        #region Methods processing check next page

        /// <summary>
        /// 
        /// </summary>
        /// <param name="response"></param>
        /// <returns></returns>
        public static bool CheckNextPage(string response)
        {
            return response.Contains("src=\"images/pagination/last-right.png\"");
        }

        #endregion

        #region Methods processing Date Time

        /// <summary>
        /// 
        /// </summary>
        /// <param name="datetime"></param>
        /// <param name="datetimeFormat"></param>
        /// <returns></returns>
        public static DateTime ConvertVnDateTimeObject(string datetime, string datetimeFormat)
        {
            string resultDate = string.Empty;
            DateTime resultDateObj = DateTime.MinValue;
            /*
             * case 1: 10 phút trước
             * case 2: 10 giờ 10 phút trước
             * case 3: Hôm qua 10:10
             * case 4: ngày 10 tháng 10 10:10
             * case 5: ngày 10 tháng 10 năm 2010 10:10
             * case 6: public on
             * case 7: đã tải lên vào 26-12-2011
             * case 8: xuất bản 02-10-2013
             */
            const string vnDay = "ngày";
            const string vnMonth = "tháng";
            const string vnYear = "năm";
            const string vnDate = "hôm qua";
            const string vnHour = "giờ";
            const string vnMinute = "phút";
            const string vnPublic = "xuất bản";
            const string enPublic = "public on";
            const string vnUploaded = "đã tải lên vào";
            const string enUploaded = "uploaded on";

            datetime = datetime.ToLower();
            if (datetime.Contains(vnYear) && datetime.Contains(vnMonth) && datetime.Contains(vnDay))
            {
                Regex regDatePostComment = new Regex(@"\d{1,2}.*\d{1,2}.*\d{2,4}.*\d{1,2}:\d{1,2}");
                Match mDatePost = regDatePostComment.Match(datetime);
                if (mDatePost.Success)
                {
                    resultDate = datetime.Replace(Regex.Replace(datetime, @"\d{1,2}.*\d{1,2}.*\d{2,4}.*\d{1,2}:\d{1,2}", string.Empty).ToString(), string.Empty);
                    int lastIndexMonth = resultDate.LastIndexOf(':') - 2;//Find date signal
                    resultDate = resultDate.Insert(lastIndexMonth, ":");// Add more symbol after date
                    resultDate = resultDate.Replace(vnMonth, ":");
                    resultDate = resultDate.Replace(vnYear, ":");
                    string[] resultDateTemp = resultDate.Split(':');
                    resultDate = int.Parse(resultDateTemp[2].Trim()) + " " + int.Parse(resultDateTemp[1].Trim()) + " " + int.Parse(resultDateTemp[0].Trim()) + " " + int.Parse(resultDateTemp[3].Trim()) + ":" + int.Parse(resultDateTemp[4].Trim());
                }
            }
            else if (datetime.Contains(vnMonth) && datetime.Contains(vnDay))
            {
                Regex regDatePostComment = new Regex(@"\d{1,2}.*\d{1,2}.*\d{1,2}:\d{1,2}");
                Match mDatePost = regDatePostComment.Match(datetime);
                if (mDatePost.Success)
                {
                    resultDate = datetime.Replace(Regex.Replace(datetime, @"\d{1,2}.*\d{1,2}.*\d{1,2}:\d{1,2}", string.Empty).ToString(), string.Empty);
                    int lastIndexMonth = resultDate.LastIndexOf(':') - 2;//Find date signal
                    resultDate = resultDate.Insert(lastIndexMonth, ":");// Add more symbol after date
                    string[] resultDateTemp = resultDate.Replace(vnMonth, ":").Split(':');
                    resultDate = DateTime.Now.Year + " " + int.Parse(resultDateTemp[1].Trim()) + " " + int.Parse(resultDateTemp[0].Trim()) + " " + int.Parse(resultDateTemp[2].Trim()) + ":" + int.Parse(resultDateTemp[3].Trim());
                }
            }
            else if (datetime.Contains(vnDate))
            {
                Regex regDatePostComment = new Regex(@"\d{1,2}:\d{1,2}");
                Match mDatePost = regDatePostComment.Match(datetime);
                if (mDatePost.Success)
                {
                    resultDate = datetime.Replace(Regex.Replace(datetime, @"\d{1,2}:\d{1,2}", string.Empty).ToString(), string.Empty);
                }
                if (!string.IsNullOrEmpty(resultDate))
                {
                    DateTime currentDate = DateTime.Now;
                    DateTime yesterday = currentDate.AddDays(-1);
                    resultDate = yesterday.Year + "/" + yesterday.Month + "/" + yesterday.Day + " " + resultDate;
                }
            }
            else if (datetime.Contains(vnHour) && datetime.Contains(vnMinute))
            {
                Regex regDatePostComment = new Regex(@"[\d{1,2}].*[\d{1,2}]");
                Match mDatePost = regDatePostComment.Match(datetime);
                if (mDatePost.Success)
                {
                    string resultDateTemp = datetime.Replace(Regex.Replace(datetime, @"[\d{1,2}].*[\d{1,2}]", string.Empty).ToString(), string.Empty);
                    string[] hourMinute = resultDateTemp.Replace(vnHour, ":").Split(':');
                    int minute = int.Parse(hourMinute[0]) * 60 + int.Parse(hourMinute[1]);
                    DateTime currentDate = DateTime.Now;
                    DateTime postDate = currentDate.AddMinutes(-minute);
                    resultDate = postDate.Year + "/" + postDate.Month + "/" + postDate.Day + " " + postDate.Hour + ":" + postDate.Minute;
                }
            }
            else if (datetime.Contains(vnMinute))
            {
                Regex regDatePostComment = new Regex(@"\d{1,2}");
                Match mDatePost = regDatePostComment.Match(datetime);
                if (mDatePost.Success)
                {
                    resultDate = datetime.Replace(Regex.Replace(datetime, @"\d{1,2}", string.Empty).ToString(), string.Empty);
                }
                if (!string.IsNullOrEmpty(resultDate))
                {
                    if (int.Parse(resultDate) == 60)
                        resultDate = DateTime.Now.Year + "/" + DateTime.Now.Month + "/" + DateTime.Now.Day + " " + DateTime.Now.AddHours(1).Hour;
                    else
                        resultDate = DateTime.Now.Year + "/" + DateTime.Now.Month + "/" + DateTime.Now.Day + " " + DateTime.Now.Hour + ":" + resultDate;
                }
            }
            else if (datetime.Contains(vnPublic))
            {
                datetime = datetime.Replace(vnPublic, string.Empty).Trim();
                resultDate = datetime;
            }
            else if (datetime.Contains(vnUploaded))
            {
                datetime = datetime.Replace(vnUploaded, string.Empty).Trim();
                resultDate = datetime;
            }
            else if (datetime.Contains(enPublic))
            {
                datetime = datetime.Replace(enPublic, string.Empty).Trim();
                DateTime postDateTemp = DateTime.ParseExact(datetime, "MMM-dd-yyyy", null);
            }
            else if (datetime.Contains(enUploaded))
            {
                datetime = datetime.Replace(enUploaded, string.Empty).Trim();
                DateTime postDateTemp = DateTime.ParseExact(datetime, "MMM-dd-yyyy", null);
            }
            try
            {
                if (!string.IsNullOrEmpty(resultDate))
                {
                    resultDateObj = ConvertToDateTimeObject(resultDate, datetimeFormat);
                }
                return resultDateObj;
            }
            catch (Exception ex)
            {

                return DateTime.MinValue;
            }
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="datetime"></param>
        /// <param name="datetimeFormat"></param>
        /// <returns></returns>
        public static DateTime ConvertGMTDateTimeObject(string datetime, string datetimeFormat)
        {
            try
            {
                DateTime objDate = ConvertToDateTimeObject(datetime, datetimeFormat);
                return objDate.ToUniversalTime();
            }
            catch (Exception ex)
            {
                return DateTime.MinValue;
            }
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="datetime"></param>
        /// <param name="datetimeFormat"></param>
        /// <returns></returns>
        public static DateTime ConvertToDateTimeObject(string datetime, string datetimeFormat)
        {
            try
            {
                if (!string.IsNullOrEmpty(datetime))
                {
                    DateTimeFormatInfo dtfi = new DateTimeFormatInfo();
                    dtfi.ShortDatePattern = datetimeFormat;
                    dtfi.DateSeparator = GetDateSeparator(dtfi.ShortDatePattern);
                    DateTime objDate = DateTime.MinValue;
                    if (dtfi.DateSeparator != string.Empty)
                    {
                        objDate = Convert.ToDateTime(datetime, dtfi);
                    }
                    else
                    {
                        objDate = DateTime.ParseExact(datetime,
                                        dtfi.ShortDatePattern,
                                        CultureInfo.InvariantCulture,
                                        DateTimeStyles.None);
                    }
                    return objDate;
                }
                else
                {
                    return DateTime.MinValue;
                }
            }
            catch (Exception ex)
            {
                //DataFeedLogger.WriteError("Error:", ex);
                return DateTime.MinValue;
            }
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="dateTimeFormat"></param>
        /// <returns></returns>
        public static string GetDateSeparator(string dateTimeFormat)
        {
            string chrResult = "";
            dateTimeFormat = dateTimeFormat.ToLower();
            char chr;
            foreach (char t in dateTimeFormat)
            {
                chr = t;
                if (chr != 'd' && chr != 'm' && chr != 'y')
                {
                    chrResult = chr.ToString();
                    break;
                }
            }
            return chrResult;
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="datetime"></param>
        /// <returns></returns>
        public static DateTime ToCETDateTime(DateTime datetime)
        {
            datetime = DateTime.SpecifyKind(datetime, DateTimeKind.Unspecified);
            return TimeZoneInfo.ConvertTime(datetime, GMTTimezoneInfo, CETTimezoneInfo);
        }

        #endregion

        #region Methods Read/Write XML file

        /// <summary>
        /// 
        /// </summary>
        /// <returns></returns>
        public static void SaveFailAnnouncement(Announcement ann)
        {

            Monitor.Enter(lockInsertNews);
            try
            {
                rely.Execute(() =>
                {
                    SaveAnnoucementToXml(ann);
                });
            }
            finally
            {
                Monitor.Exit(lockInsertNews);
            }
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="id"></param>
        public static void DeleteFailAnnouncement(long id)
        {
            Monitor.Enter(lockInsertNews);
            try
            {
                rely.Execute(() =>
                {
                    RemoveFailAnnouncementFromXml(id); ;
                });
            }
            finally
            {
                Monitor.Exit(lockInsertNews);
            }
        }

        /// <summary>
        /// 
        /// </summary>
        /// <returns></returns>
        public static List<Announcement> LoadFailAnnouncement()
        {
            List<Announcement> lstAnnouncement = new List<Announcement>();
            string sPath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
            try
            {
                if (File.Exists(Path.Combine(sPath, FailInsertingNewsFileName)))
                {
                    XmlDocument doc = new XmlDocument();
                    doc.Load(Path.Combine(sPath, FailInsertingNewsFileName));
                    XmlNodeList Annoucements = doc.GetElementsByTagName("Annoucement");

                    foreach (XmlNode anc in Annoucements)
                    {
                        Announcement item = new Announcement();

                        XmlAttribute attrFailID = anc.Attributes["id"];
                        item.FailId = long.Parse(attrFailID.Value);

                        XmlAttribute attrDbId = anc.Attributes["dbid"];
                        if (attrDbId != null)
                            item.Id = long.Parse(attrDbId.Value);
                        else
                            item.Id = 0;

                        XmlNode Date = anc.SelectSingleNode("./Date");
                        item.DateTime = DateTime.Parse(Date.InnerText);

                        XmlNode DateNumberic = anc.SelectSingleNode("./DateNumberic");
                        item.DateNumberic = long.Parse(DateNumberic.InnerText);

                        XmlNode SourceCode = anc.SelectSingleNode("./CompanyCode");
                        item.SourceCode = SourceCode.InnerText;

                        XmlNode LangId = anc.SelectSingleNode("./LangId");
                        item.LangId = int.Parse(LangId.InnerText);

                        XmlNode Title = anc.SelectSingleNode("./Title");
                        item.Title = Title.FirstChild.Value;

                        XmlNode MessageTypeId = anc.SelectSingleNode("./MessageTypeId");
                        item.MessageTypeId = ((new Regex(@"^\d+$")).IsMatch(MessageTypeId.InnerText)) ? int.Parse(MessageTypeId.InnerText) : (int?)null;

                        XmlNode Attachment = anc.SelectSingleNode("./Attachment");
                        item.Attachment = bool.Parse(Attachment.InnerText);

                        XmlNode SourceId = anc.SelectSingleNode("./SourceId");
                        item.SourceId = int.Parse(SourceId.InnerText);

                        /*XmlNode OriginalId = anc.SelectSingleNode("./OriginalId");
                        item.OriginalId = int.Parse(OriginalId.InnerText);

                        XmlNode Pressrelease = anc.SelectSingleNode("./Pressrelease");
                        item.Pressrelease = bool.Parse(Pressrelease.InnerText);                        

                        XmlNode PressrelaseType = anc.SelectSingleNode("./PressrelaseType");
                        item.PressrelaseType = PressrelaseType.InnerText;                        

                        XmlNode FormType = anc.SelectSingleNode("./FormType");
                        if (FormType != null)
                            item.FormType = FormType.InnerText;

                        XmlNode Hidden = anc.SelectSingleNode("./Hidden");
                        item.Hidden = bool.Parse(Hidden.InnerText);*/

                        XmlNode Link = anc.SelectSingleNode("./Link");
                        item.Link = Link.InnerText;

                        XmlNode LocalFileName = anc.SelectSingleNode("./LocalFileName");
                        if (LocalFileName != null)
                            item.PhysicalFileName = LocalFileName.InnerText;

                        if (string.IsNullOrEmpty(item.PhysicalFileName))
                            item.PhysicalFileName = GetFilename(item.Link);

                        XmlNode InsertedDate = anc.SelectSingleNode("./InsertedDate");
                        item.InsertedDate = DateTime.Parse(InsertedDate.InnerText);

                        XmlNodeList AttachmentLinks = anc.SelectNodes("./AttachmentLinks/url");
                        if (AttachmentLinks != null && AttachmentLinks.Count > 0)
                        {
                            List<string> lstAttachmentLinks = new List<string>();
                            foreach (XmlNode url in AttachmentLinks)
                            {
                                lstAttachmentLinks.Add(url.InnerText);
                            }
                            item.AttachmentLinks = lstAttachmentLinks;
                        }

                        lstAnnouncement.Add(item);
                    }
                }
            }
            catch (Exception ex)
            {
                Logger.WriteError("Cannot load Fail Announcement from XML!", ex);
            }
            return lstAnnouncement;
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="ann"></param>
        private static void SaveAnnoucementToXml(Announcement ann)
        {
            try
            {
                string sPath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
                string fileName = Path.Combine(sPath, FailInsertingNewsFileName);
                // Create the XmlDocument.
                XmlDocument doc = new XmlDocument();
                XmlNode elRoot = null;
                //Check existed fail log file
                if (File.Exists(fileName))
                {
                    doc.Load(fileName);
                    if (doc.DocumentElement == null)
                    {
                        doc.LoadXml("<FailAnnoucements></FailAnnoucements>");
                    }
                }
                else
                {
                    //elRoot = doc.CreateNode(XmlNodeType.Element, "FailAnnoucements", string.Empty);
                    doc.LoadXml("<FailAnnoucements></FailAnnoucements>");
                    //doc.AppendChild(elRoot);

                }

                elRoot = doc.DocumentElement;

                // Check the announcement link instead of ID to avoid duplicated element
                // TODO: is there any syntax error if the "Link" has apostrophe ('), Ampersand (&), Left and right angle bracket (<>), Straight quotation mark (")???
                XmlNode linkNode = elRoot.SelectSingleNode("Annoucement/Link[translate(text(), 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz')=translate('" + ann.Link + "', 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', 'abcdefghijklmnopqrstuvwxyz')]");

                //Check existed fail announcement in xml log.
                //If existed then check RequestCount attribute > 10. If true then remove from XML log else increase ReqeustCount attribute value.
                XmlNode announcement = null;
                if (linkNode == null)
                {
                    announcement = elRoot.SelectSingleNode("Annoucement[@id='" + ann.FailId.ToString() + "']");
                    if (announcement == null)
                        announcement = elRoot.SelectSingleNode("Annoucement[@dbid='" + ann.Id.ToString() + "']");
                }
                else
                {
                    announcement = linkNode.ParentNode;
                }

                if (announcement != null)
                {
                    if (announcement.Attributes["id"] != null
                        && announcement.Attributes["id"].Value != ann.FailId.ToString())
                    {
                        XmlNode AttachmentLinks = doc.SelectSingleNode("./AttachmentLinks");
                        if (AttachmentLinks != null)
                        {
                            bool isLinkExisted = false;
                            foreach (XmlNode link in AttachmentLinks.ChildNodes)
                            {
                                if (link.InnerText == ann.Link)
                                {
                                    isLinkExisted = true;
                                    break;
                                }
                            }
                            if (!isLinkExisted)
                            {
                                XmlNode url = doc.CreateNode(XmlNodeType.Element, "url", string.Empty);
                                url.InnerText = ann.Link;
                                AttachmentLinks.AppendChild(url);
                            }
                        }
                    }
                    else
                    {
                        XmlAttribute retryCountAttr = announcement.Attributes["Retry"];
                        if (retryCountAttr != null)
                        {
                            long retryCount = long.Parse(retryCountAttr.Value);
                            if (retryCount == 10)
                            {
                                //Remove announcement from xml log
                                RemoveFailAnnouncementFromXml(ann.FailId);
                            }
                            else
                            {
                                retryCount++;
                                retryCountAttr.Value = retryCount.ToString();

                                XmlNode AttachmentLinks = doc.SelectSingleNode("./AttachmentLinks");
                                if (AttachmentLinks != null)
                                {
                                    bool isLinkExisted = false;
                                    foreach (XmlNode link in AttachmentLinks.ChildNodes)
                                    {
                                        if (link.InnerText == ann.Link)
                                        {
                                            isLinkExisted = true;
                                            break;
                                        }
                                    }
                                    if (!isLinkExisted)
                                    {
                                        XmlNode url = doc.CreateNode(XmlNodeType.Element, "url", string.Empty);
                                        url.InnerText = ann.Link;
                                        AttachmentLinks.AppendChild(url);
                                    }
                                }
                                else
                                {
                                    AttachmentLinks = doc.CreateNode(XmlNodeType.Element, "AttachmentLinks", string.Empty);
                                    foreach (string link in ann.AttachmentLinks)
                                    {
                                        XmlNode url = doc.CreateNode(XmlNodeType.Element, "url", string.Empty);
                                        url.InnerText = link;
                                        AttachmentLinks.AppendChild(url);
                                    }
                                    announcement.AppendChild(AttachmentLinks);
                                }
                            }
                        }
                        else
                        {
                            //Create RequestCount attribute
                            XmlAttribute newRetryCountAttr = doc.CreateAttribute("Retry");
                            newRetryCountAttr.Value = "2";
                            announcement.Attributes.Append(newRetryCountAttr);
                        }
                    }
                }
                else
                {
                    XmlNode newSource = doc.CreateNode(XmlNodeType.Element, "Annoucement", string.Empty);
                    XmlAttribute idAttr = doc.CreateAttribute("id");
                    idAttr.Value = ann.FailId.ToString();
                    newSource.Attributes.Append(idAttr);
                    if (ann.Id > 0)
                    {
                        XmlAttribute dbidAttr = doc.CreateAttribute("dbid");
                        dbidAttr.Value = ann.Id.ToString();
                        newSource.Attributes.Append(dbidAttr);
                    }

                    XmlNode Date = doc.CreateNode(XmlNodeType.Element, "Date", string.Empty);
                    Date.InnerText = ann.DateTime.ToString();
                    newSource.AppendChild(Date);

                    XmlNode DateNumberic = doc.CreateNode(XmlNodeType.Element, "DateNumberic", string.Empty);
                    DateNumberic.InnerText = ann.DateNumberic.ToString();
                    newSource.AppendChild(DateNumberic);

                    XmlNode SourceCode = doc.CreateNode(XmlNodeType.Element, "SourceCode", string.Empty);
                    SourceCode.InnerText = ann.SourceCode;
                    newSource.AppendChild(SourceCode);

                    XmlNode LangId = doc.CreateNode(XmlNodeType.Element, "LangId", string.Empty);
                    LangId.InnerText = ann.LangId.ToString();
                    newSource.AppendChild(LangId);

                    XmlNode Title = doc.CreateNode(XmlNodeType.Element, "Title", string.Empty);
                    XmlNode titleData = doc.CreateNode(XmlNodeType.CDATA, string.Empty, string.Empty);
                    titleData.Value = ann.Title;
                    Title.AppendChild(titleData);
                    newSource.AppendChild(Title);

                    XmlNode MessageTypeId = doc.CreateNode(XmlNodeType.Element, "MessageTypeId", string.Empty);
                    MessageTypeId.InnerText = ann.MessageTypeId != null ? ann.MessageTypeId.ToString() : "";
                    newSource.AppendChild(MessageTypeId);

                    XmlNode Attachment = doc.CreateNode(XmlNodeType.Element, "Attachment", string.Empty);
                    Attachment.InnerText = ann.Attachment.ToString();
                    newSource.AppendChild(Attachment);

                    XmlNode SourceId = doc.CreateNode(XmlNodeType.Element, "SourceId", string.Empty);
                    SourceId.InnerText = ann.SourceId.ToString();
                    newSource.AppendChild(SourceId);

                    /*XmlNode OriginalId = doc.CreateNode(XmlNodeType.Element, "OriginalId", string.Empty);
                    OriginalId.InnerText = ann.OriginalId.ToString();
                    newSource.AppendChild(OriginalId);

                    XmlNode Pressrelease = doc.CreateNode(XmlNodeType.Element, "Pressrelease", string.Empty);
                    Pressrelease.InnerText = ann.Pressrelease.ToString();
                    newSource.AppendChild(Pressrelease);                    

                    XmlNode PressrelaseType = doc.CreateNode(XmlNodeType.Element, "PressrelaseType", string.Empty);
                    PressrelaseType.InnerText = ann.PressrelaseType ?? "";
                    newSource.AppendChild(PressrelaseType);                    
                      
                    XmlNode Hidden = doc.CreateNode(XmlNodeType.Element, "Hidden", string.Empty);
                    Hidden.InnerText = ann.Hidden.ToString();
                    newSource.AppendChild(Hidden); 

                    XmlNode FormType = doc.CreateNode(XmlNodeType.Element, "FormType", string.Empty);
                    FormType.InnerText = ann.FormType;
                    newSource.AppendChild(FormType);*/

                    XmlNode InsertedDate = doc.CreateNode(XmlNodeType.Element, "InsertedDate", string.Empty);
                    InsertedDate.InnerText = ann.InsertedDate.ToString();
                    newSource.AppendChild(InsertedDate);

                    XmlNode Link = doc.CreateNode(XmlNodeType.Element, "Link", string.Empty);
                    Link.InnerText = ann.Link;
                    newSource.AppendChild(Link);

                    XmlNode LocalFileName = doc.CreateNode(XmlNodeType.Element, "LocalFileName", string.Empty);
                    LocalFileName.InnerText = ann.PhysicalFileName;
                    newSource.AppendChild(LocalFileName);

                    if (ann.AttachmentLinks != null && ann.AttachmentLinks.Count > 0)
                    {
                        XmlNode AttachmentLinks = doc.CreateNode(XmlNodeType.Element, "AttachmentLinks", string.Empty);
                        foreach (string link in ann.AttachmentLinks)
                        {
                            XmlNode url = doc.CreateNode(XmlNodeType.Element, "url", string.Empty);
                            url.InnerText = link;
                            AttachmentLinks.AppendChild(url);
                        }
                        newSource.AppendChild(AttachmentLinks);
                    }

                    elRoot.AppendChild(newSource);
                }

                XmlTextWriter writer = new XmlTextWriter(Path.Combine(sPath, FailInsertingNewsFileName), Encoding.UTF8);
                writer.Formatting = Formatting.Indented;

                try
                {
                    doc.Save(writer);
                }
                finally
                {
                    writer.Close();
                }
            }
            catch (Exception ex)
            {
                Logger.WriteError("Could not save fail Announcement to XML!", ex);
                throw ex;
            }
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="id"></param>
        private static void RemoveFailAnnouncementFromXml(long id)
        {
            string sPath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
            if (File.Exists(Path.Combine(sPath, FailInsertingNewsFileName)))
            {
                try
                {
                    XmlDocument doc = new XmlDocument();
                    doc.Load(Path.Combine(sPath, FailInsertingNewsFileName));
                    //Get element have id to delete
                    XmlNode announcement = doc.DocumentElement.SelectSingleNode("Annoucement[@id='" + id.ToString() + "']");
                    if (announcement != null)
                    {
                        announcement.ParentNode.RemoveChild(announcement);
                        StreamWriter outStream = System.IO.File.CreateText(Path.Combine(sPath, FailInsertingNewsFileName));
                        doc.Save(outStream);
                        outStream.Close();
                    }
                }
                catch (Exception ex)
                {
                    Logger.WriteError("Cannot delete fail Announcement from XML!", ex);
                }
            }
        }

        /// <summary>
        /// 
        /// </summary>
        /// <returns></returns>
        public static List<string> IgnoreAnnouncementTitles()
        {
            try
            {
                string sPath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
                // Create the XmlDocument.
                XmlDocument doc = new XmlDocument();
                List<string> lstIgnoreTitle = new List<string>();
                //Check existed fail log file
                if (File.Exists(Path.Combine(sPath, IgnoreAnnouncementFileName)))
                {
                    doc.Load(Path.Combine(sPath, IgnoreAnnouncementFileName));
                    XmlNodeList titles = doc.GetElementsByTagName("title");
                    foreach (XmlNode title in titles)
                    {
                        XmlCDataSection cdataSection = title.FirstChild as XmlCDataSection;
                        if (cdataSection != null)
                        {
                            lstIgnoreTitle.Add(cdataSection.Value.ToLower());
                        }
                        else
                        {
                            lstIgnoreTitle.Add(title.InnerText.ToLower());
                        }
                    }
                }
                return lstIgnoreTitle;
            }
            catch (Exception ex)
            {
                Logger.WriteError("Cannot load Ignore Announcement from XML!", ex);
                return new List<string>();
            }
        }

        public static List<string> IgnoreAnnouncements
        {
            get;
            set;
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="ann"></param>
        public static void SaveDataToXml(Announcement ann)
        {
            if (ann.SourceCode.ToLower() == SourceCodes.CLASSIFIED_AD.ToString().ToLower())
                WriteDataChotot2Xml(ann);
            else if (ann.SourceCode.ToLower() == SourceCodes.FORUM.ToString().ToLower())
                WriteDataForum2Xml(ann);
            else if (ann.SourceCode.ToLower() == SourceCodes.INFORMATION_INSTRUCTION.ToString().ToLower())
                WriteDataInstructionInfo2Xml(ann);
            else
                WriteData2XmlCommon(ann);
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="ann"></param>
        private static void WriteData2XmlCommon(Announcement ann)
        {
            try
            {
                string date = DateTime.Now.Year + "-" + DateTime.Now.Month.ToString("00") + "-" + DateTime.Now.Day.ToString("00") + "-" + DateTime.Now.Date.Hour.ToString("00");
                string folder = Utility.GetSourceCodeFolderName(ann.SourceId);
                string directoryPath = Path.Combine(DownloadedFolderPath, ann.SourceCode, folder, date);
                if (!Directory.Exists(directoryPath))
                    Directory.CreateDirectory(directoryPath);

                string file = MiscUtility.ProcessString(UnicodeUtility.UnicodeToNoSign(ann.Title)) + ".xml";
                string fileName = Path.Combine(directoryPath, file);
                // Create the XmlDocument.
                XmlDocument doc = new XmlDocument();
                XmlNode elRoot = null;
                doc.LoadXml("<Topics></Topics>");
                elRoot = doc.DocumentElement;


                XmlNode newSource = null;
                newSource = doc.CreateNode(XmlNodeType.Element, "Topic", string.Empty);

                if (ann.DateTime != DateTime.MinValue)
                {
                    XmlNode Date = doc.CreateNode(XmlNodeType.Element, "DateTime", string.Empty);
                    Date.InnerText = ann.DateTime.ToString();
                    newSource.AppendChild(Date);
                }

                if (ann.DateTime != DateTime.MinValue)
                {
                    XmlNode DateNumberic = doc.CreateNode(XmlNodeType.Element, "DateNumberic", string.Empty);
                    DateNumberic.InnerText = ann.DateNumberic.ToString();
                    newSource.AppendChild(DateNumberic);
                }

                if (!string.IsNullOrEmpty(ann.SourceCode))
                {
                    XmlNode SourceCode = doc.CreateNode(XmlNodeType.Element, "SourceCode", string.Empty);
                    SourceCode.InnerText = ann.SourceCode;
                    newSource.AppendChild(SourceCode);
                }

                if (ann.LangId != 0)
                {
                    XmlNode LangId = doc.CreateNode(XmlNodeType.Element, "LangId", string.Empty);
                    LangId.InnerText = ann.LangId.ToString();
                    newSource.AppendChild(LangId);
                }

                if (!string.IsNullOrEmpty(ann.Title))
                {
                    XmlNode Title = doc.CreateNode(XmlNodeType.Element, "Title", string.Empty);
                    XmlNode titleData = doc.CreateNode(XmlNodeType.CDATA, string.Empty, string.Empty);
                    titleData.Value = ann.Title;
                    Title.AppendChild(titleData);
                    newSource.AppendChild(Title);
                }

                if (!string.IsNullOrEmpty(ann.Message))
                {
                    XmlNode Message = doc.CreateNode(XmlNodeType.Element, "Message", string.Empty);
                    XmlNode MessageData = doc.CreateNode(XmlNodeType.CDATA, string.Empty, string.Empty);
                    MessageData.Value = ann.Message;
                    Message.AppendChild(MessageData);
                    newSource.AppendChild(Message);
                }

                /*if (ann.MessageTypeId.HasValue)
                {
                    XmlNode MessageTypeId = doc.CreateNode(XmlNodeType.Element, "MessageTypeId", string.Empty);
                    MessageTypeId.InnerText = ann.MessageTypeId != null ? ann.MessageTypeId.ToString() : "";
                    newSource.AppendChild(MessageTypeId);
                }*/
                if (!ann.Attachment)
                {
                    XmlNode Attachment = doc.CreateNode(XmlNodeType.Element, "Attachment", string.Empty);
                    Attachment.InnerText = ann.Attachment.ToString();
                    newSource.AppendChild(Attachment);
                }

                if (ann.SourceId != 0)
                {
                    XmlNode SourceId = doc.CreateNode(XmlNodeType.Element, "SourceId", string.Empty);
                    SourceId.InnerText = ann.SourceId.ToString();
                    newSource.AppendChild(SourceId);
                }

                // Add Restaurant information 
                if (ann.SocialNetwork.RestaurantInfo.ContactInfo != null)
                {
                    if (ann.SocialNetwork.RestaurantInfo.ContactInfo.Address != null)
                    {
                        XmlNode RestaurantAddress = doc.CreateNode(XmlNodeType.Element, "Address", string.Empty);
                        RestaurantAddress.InnerText = ann.SocialNetwork.RestaurantInfo.ContactInfo.Address.ToString();
                        newSource.AppendChild(RestaurantAddress);
                    }

                    if (ann.SocialNetwork.RestaurantInfo.ContactInfo.HomePhone != null)
                    {
                        XmlNode RestaurantPhone = doc.CreateNode(XmlNodeType.Element, "HomePhone", string.Empty);
                        RestaurantPhone.InnerText = ann.SocialNetwork.RestaurantInfo.ContactInfo.HomePhone.ToString();
                        newSource.AppendChild(RestaurantPhone);
                    }
                }

                if (ann.SocialNetwork.ScreenName != null)
                {
                    XmlNode UserName = doc.CreateNode(XmlNodeType.Element, "UserName", string.Empty);
                    UserName.InnerText = ann.SocialNetwork.ScreenName.ToString();
                    newSource.AppendChild(UserName);
                }

                if (ann.Link != null)
                {
                    XmlNode Link = doc.CreateNode(XmlNodeType.Element, "Link", string.Empty);
                    Link.InnerText = ann.Link;
                    newSource.AppendChild(Link);
                }

                if (ann.PhysicalFileName != null)
                {
                    XmlNode PhysicalFileName = doc.CreateNode(XmlNodeType.Element, "PhysicalFileName", string.Empty);
                    PhysicalFileName.InnerText = ann.PhysicalFileName;
                    newSource.AppendChild(PhysicalFileName);
                }
                if (ann.InsertedDate != null)
                {
                    XmlNode InsertedDate = doc.CreateNode(XmlNodeType.Element, "InsertedDate", string.Empty);
                    InsertedDate.InnerText = ann.InsertedDate.ToString();
                    newSource.AppendChild(InsertedDate);
                }

                if (ann.AttachmentLinks != null && ann.AttachmentLinks.Count > 0)
                {
                    XmlNode AttachmentLinks = doc.CreateNode(XmlNodeType.Element, "AttachmentLinks", string.Empty);
                    foreach (string link in ann.AttachmentLinks)
                    {
                        XmlNode url = doc.CreateNode(XmlNodeType.Element, "url", string.Empty);
                        url.InnerText = link;
                        AttachmentLinks.AppendChild(url);
                    }
                    newSource.AppendChild(AttachmentLinks);
                }

                // Add comment's Forum 
                if (ann.SocialNetwork.Comments != null && ann.SocialNetwork.Comments.Count > 0)
                {
                    XmlNode comments = doc.CreateNode(XmlNodeType.Element, "Comments", string.Empty);
                    foreach (Comment cm in ann.SocialNetwork.Comments)
                    {
                        XmlNode comment = doc.CreateNode(XmlNodeType.Element, "Comment", string.Empty);

                        if (cm.DateTime != null)
                        {
                            XmlNode dateTime = doc.CreateNode(XmlNodeType.Element, "DateTime", string.Empty);
                            dateTime.InnerText = cm.DateTime.ToString();
                            comment.AppendChild(dateTime);
                        }

                        if (cm.DateNumberic != 0)
                        {
                            XmlNode dateNumeric = doc.CreateNode(XmlNodeType.Element, "DateNumberic", string.Empty);
                            dateNumeric.InnerText = cm.DateNumberic.ToString();
                            comment.AppendChild(dateNumeric);
                        }

                        if (!string.IsNullOrEmpty(cm.UserName))
                        {
                            XmlNode userName = doc.CreateNode(XmlNodeType.Element, "UserName", string.Empty);
                            userName.InnerText = cm.UserName.ToString();
                            comment.AppendChild(userName);
                        }

                        if (!string.IsNullOrEmpty(cm.CommentContent))
                        {
                            XmlNode commentContent = doc.CreateNode(XmlNodeType.Element, "CommentContent", string.Empty);
                            XmlNode commentContentData = doc.CreateNode(XmlNodeType.CDATA, string.Empty, string.Empty);
                            commentContentData.Value = cm.CommentContent;
                            commentContent.AppendChild(commentContentData);
                            comment.AppendChild(commentContent);
                        }

                        if (cm.AttachmentLinks != null && cm.AttachmentLinks.Count > 0)
                        {
                            XmlNode commentAttachmentLinks = doc.CreateNode(XmlNodeType.Element, "AttachmentLinks", string.Empty);
                            foreach (string link in ann.AttachmentLinks)
                            {
                                XmlNode url = doc.CreateNode(XmlNodeType.Element, "url", string.Empty);
                                url.InnerText = link;
                                commentAttachmentLinks.AppendChild(url);
                            }
                            newSource.AppendChild(commentAttachmentLinks);
                        }
                        comments.AppendChild(comment);
                    }
                    newSource.AppendChild(comments);
                }

                elRoot.AppendChild(newSource);

                XmlTextWriter writer = new XmlTextWriter(Path.Combine(directoryPath, file), Encoding.UTF8);
                writer.Formatting = Formatting.Indented;

                try
                {
                    doc.Save(writer);
                }
                finally
                {
                    writer.Close();
                }
            }
            catch (Exception ex)
            {
                Logger.WriteError("Could not save data of Forum Announcement to XML!", ex);
                throw ex;
            }
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="ann"></param>
        private static void WriteDataInstructionInfo2Xml(Announcement ann)
        {
            try
            {
                string date = DateTime.Now.Year + "-" + DateTime.Now.Month.ToString("00") + "-" + DateTime.Now.Day.ToString("00") + "-" + DateTime.Now.Date.Hour.ToString("00");
                string folder = Utility.GetSourceCodeFolderName(ann.SourceId);
                string directoryPath = Path.Combine(DownloadedFolderPath, ann.SourceCode, folder, date);
                if (!Directory.Exists(directoryPath))
                    Directory.CreateDirectory(directoryPath);

                string file = MiscUtility.ProcessString(UnicodeUtility.UnicodeToNoSign(ann.Title)) + ".xml";
                string fileName = Path.Combine(directoryPath, file);
                // Create the XmlDocument.
                XmlDocument doc = new XmlDocument();
                XmlNode elRoot = null;
                doc.LoadXml("<Places></Places>");
                elRoot = doc.DocumentElement;


                XmlNode newSource = null;
                newSource = doc.CreateNode(XmlNodeType.Element, "Place", string.Empty);

                if (!string.IsNullOrEmpty(ann.Title))
                {
                    XmlNode Title = doc.CreateNode(XmlNodeType.Element, "Title", string.Empty);
                    XmlNode titleData = doc.CreateNode(XmlNodeType.CDATA, string.Empty, string.Empty);
                    titleData.Value = ann.Title;
                    Title.AppendChild(titleData);
                    newSource.AppendChild(Title);
                }

                if (!string.IsNullOrEmpty(ann.Message))
                {
                    XmlNode Message = doc.CreateNode(XmlNodeType.Element, "Message", string.Empty);
                    XmlNode MessageData = doc.CreateNode(XmlNodeType.CDATA, string.Empty, string.Empty);
                    MessageData.Value = ann.Message;
                    Message.AppendChild(MessageData);
                    newSource.AppendChild(Message);
                }

                // Add Restaurant information 
                if (ann.SocialNetwork.RestaurantInfo.ContactInfo != null)
                {
                    if (ann.SocialNetwork.RestaurantInfo.ContactInfo.Address != null)
                    {
                        XmlNode RestaurantAddress = doc.CreateNode(XmlNodeType.Element, "Address", string.Empty);
                        RestaurantAddress.InnerText = ann.SocialNetwork.RestaurantInfo.ContactInfo.Address.ToString();
                        newSource.AppendChild(RestaurantAddress);
                    }

                    if (ann.SocialNetwork.RestaurantInfo.ContactInfo.HomePhone != null)
                    {
                        XmlNode RestaurantPhone = doc.CreateNode(XmlNodeType.Element, "HomePhone", string.Empty);
                        RestaurantPhone.InnerText = ann.SocialNetwork.RestaurantInfo.ContactInfo.HomePhone.ToString();
                        newSource.AppendChild(RestaurantPhone);
                    }
                }

                if (ann.SocialNetwork.ScreenName != null)
                {
                    XmlNode UserName = doc.CreateNode(XmlNodeType.Element, "UserName", string.Empty);
                    UserName.InnerText = ann.SocialNetwork.ScreenName.ToString();
                    newSource.AppendChild(UserName);
                }

                if (ann.Link != null)
                {
                    XmlNode Link = doc.CreateNode(XmlNodeType.Element, "Link", string.Empty);
                    Link.InnerText = ann.Link;
                    newSource.AppendChild(Link);
                }

                if (ann.InsertedDate != null)
                {
                    XmlNode InsertedDate = doc.CreateNode(XmlNodeType.Element, "InsertedDate", string.Empty);
                    InsertedDate.InnerText = ann.InsertedDate.ToString();
                    newSource.AppendChild(InsertedDate);
                }

                if (ann.AttachmentLinks != null && ann.AttachmentLinks.Count > 0)
                {
                    XmlNode AttachmentLinks = doc.CreateNode(XmlNodeType.Element, "AttachmentLinks", string.Empty);
                    foreach (string link in ann.AttachmentLinks)
                    {
                        XmlNode url = doc.CreateNode(XmlNodeType.Element, "url", string.Empty);
                        url.InnerText = link;
                        AttachmentLinks.AppendChild(url);
                    }
                    newSource.AppendChild(AttachmentLinks);
                }

                // Add comment's Forum 
                if (ann.SocialNetwork.Comments != null && ann.SocialNetwork.Comments.Count > 0)
                {
                    XmlNode comments = doc.CreateNode(XmlNodeType.Element, "Comments", string.Empty);
                    foreach (Comment cm in ann.SocialNetwork.Comments)
                    {
                        XmlNode comment = doc.CreateNode(XmlNodeType.Element, "Comment", string.Empty);

                        if (cm.DateTime != null)
                        {
                            XmlNode dateTime = doc.CreateNode(XmlNodeType.Element, "DateTime", string.Empty);
                            dateTime.InnerText = cm.DateTime.ToString();
                            comment.AppendChild(dateTime);
                        }

                        if (cm.DateNumberic != 0)
                        {
                            XmlNode dateNumeric = doc.CreateNode(XmlNodeType.Element, "DateNumberic", string.Empty);
                            dateNumeric.InnerText = cm.DateNumberic.ToString();
                            comment.AppendChild(dateNumeric);
                        }

                        if (!string.IsNullOrEmpty(cm.UserName))
                        {
                            XmlNode userName = doc.CreateNode(XmlNodeType.Element, "UserName", string.Empty);
                            userName.InnerText = cm.UserName.ToString();
                            comment.AppendChild(userName);
                        }

                        if (!string.IsNullOrEmpty(cm.CommentContent))
                        {
                            XmlNode commentContent = doc.CreateNode(XmlNodeType.Element, "CommentContent", string.Empty);
                            XmlNode commentContentData = doc.CreateNode(XmlNodeType.CDATA, string.Empty, string.Empty);
                            commentContentData.Value = cm.CommentContent;
                            commentContent.AppendChild(commentContentData);
                            comment.AppendChild(commentContent);
                        }

                        if (cm.AttachmentLinks != null && cm.AttachmentLinks.Count > 0)
                        {
                            XmlNode commentAttachmentLinks = doc.CreateNode(XmlNodeType.Element, "AttachmentLinks", string.Empty);
                            foreach (string link in ann.AttachmentLinks)
                            {
                                XmlNode url = doc.CreateNode(XmlNodeType.Element, "url", string.Empty);
                                url.InnerText = link;
                                commentAttachmentLinks.AppendChild(url);
                            }
                            newSource.AppendChild(commentAttachmentLinks);
                        }
                        comments.AppendChild(comment);
                    }
                    newSource.AppendChild(comments);
                }

                elRoot.AppendChild(newSource);

                XmlTextWriter writer = new XmlTextWriter(Path.Combine(directoryPath, file), Encoding.UTF8);
                writer.Formatting = Formatting.Indented;

                try
                {
                    doc.Save(writer);
                }
                finally
                {
                    writer.Close();
                }
            }
            catch (Exception ex)
            {
                Logger.WriteError("Could not save data of Forum Announcement to XML!", ex);
                throw ex;
            }
        }

        /// <summary>
        /// /
        /// </summary>
        /// <param name="ann"></param>
        private static void WriteDataForum2Xml(Announcement ann)
        {
            try
            {
                string date = DateTime.Now.Year + "-" + DateTime.Now.Month.ToString("00") + "-" + DateTime.Now.Day.ToString("00") + "-" + DateTime.Now.Date.Hour.ToString("00");
                string folder = Utility.GetSourceCodeFolderName(ann.SourceId);
                string directoryPath = Path.Combine(DownloadedFolderPath, ann.SourceCode, folder, date);
                if (!Directory.Exists(directoryPath))
                    Directory.CreateDirectory(directoryPath);

                string file = MiscUtility.ProcessString(UnicodeUtility.UnicodeToNoSign(ann.Title)) + ".xml";
                string fileName = Path.Combine(directoryPath, file);
                // Create the XmlDocument.
                XmlDocument doc = new XmlDocument();
                XmlNode elRoot = null;
                doc.LoadXml("<Topics></Topics>");
                elRoot = doc.DocumentElement;


                XmlNode newSource = null;
                newSource = doc.CreateNode(XmlNodeType.Element, "Topic", string.Empty);

                if (ann.DateTime != DateTime.MinValue)
                {
                    XmlNode Date = doc.CreateNode(XmlNodeType.Element, "DateTime", string.Empty);
                    Date.InnerText = ann.DateTime.ToString();
                    newSource.AppendChild(Date);
                }

                if (!string.IsNullOrEmpty(ann.Title))
                {
                    XmlNode Title = doc.CreateNode(XmlNodeType.Element, "Title", string.Empty);
                    XmlNode titleData = doc.CreateNode(XmlNodeType.CDATA, string.Empty, string.Empty);
                    titleData.Value = ann.Title;
                    Title.AppendChild(titleData);
                    newSource.AppendChild(Title);
                }

                if (!string.IsNullOrEmpty(ann.Message))
                {
                    XmlNode Message = doc.CreateNode(XmlNodeType.Element, "Message", string.Empty);
                    XmlNode MessageData = doc.CreateNode(XmlNodeType.CDATA, string.Empty, string.Empty);
                    MessageData.Value = ann.Message;
                    Message.AppendChild(MessageData);
                    newSource.AppendChild(Message);
                }

                if (ann.SourceId != 0)
                {
                    XmlNode SourceId = doc.CreateNode(XmlNodeType.Element, "SourceId", string.Empty);
                    SourceId.InnerText = ann.SourceId.ToString();
                    newSource.AppendChild(SourceId);
                }

                if (ann.SocialNetwork.ScreenName != null)
                {
                    XmlNode UserName = doc.CreateNode(XmlNodeType.Element, "UserName", string.Empty);
                    UserName.InnerText = ann.SocialNetwork.ScreenName.ToString();
                    newSource.AppendChild(UserName);
                }

                if (ann.Link != null)
                {
                    XmlNode Link = doc.CreateNode(XmlNodeType.Element, "Link", string.Empty);
                    Link.InnerText = ann.Link;
                    newSource.AppendChild(Link);
                }


                if (ann.InsertedDate != null)
                {
                    XmlNode InsertedDate = doc.CreateNode(XmlNodeType.Element, "InsertedDate", string.Empty);
                    InsertedDate.InnerText = ann.InsertedDate.ToString();
                    newSource.AppendChild(InsertedDate);
                }

                if (ann.AttachmentLinks != null && ann.AttachmentLinks.Count > 0)
                {
                    XmlNode AttachmentLinks = doc.CreateNode(XmlNodeType.Element, "AttachmentLinks", string.Empty);
                    foreach (string link in ann.AttachmentLinks)
                    {
                        XmlNode url = doc.CreateNode(XmlNodeType.Element, "url", string.Empty);
                        url.InnerText = link;
                        AttachmentLinks.AppendChild(url);
                    }
                    newSource.AppendChild(AttachmentLinks);
                }

                // Add comment's Forum 
                if (ann.SocialNetwork.Comments != null && ann.SocialNetwork.Comments.Count > 0)
                {
                    XmlNode comments = doc.CreateNode(XmlNodeType.Element, "Comments", string.Empty);
                    foreach (Comment cm in ann.SocialNetwork.Comments)
                    {
                        XmlNode comment = doc.CreateNode(XmlNodeType.Element, "Comment", string.Empty);

                        if (cm.DateTime != null)
                        {
                            XmlNode dateTime = doc.CreateNode(XmlNodeType.Element, "DateTime", string.Empty);
                            dateTime.InnerText = cm.DateTime.ToString();
                            comment.AppendChild(dateTime);
                        }

                        if (cm.DateNumberic != 0)
                        {
                            XmlNode dateNumeric = doc.CreateNode(XmlNodeType.Element, "DateNumberic", string.Empty);
                            dateNumeric.InnerText = cm.DateNumberic.ToString();
                            comment.AppendChild(dateNumeric);
                        }

                        if (!string.IsNullOrEmpty(cm.UserName))
                        {
                            XmlNode userName = doc.CreateNode(XmlNodeType.Element, "UserName", string.Empty);
                            userName.InnerText = cm.UserName.ToString();
                            comment.AppendChild(userName);
                        }

                        if (!string.IsNullOrEmpty(cm.CommentContent))
                        {
                            XmlNode commentContent = doc.CreateNode(XmlNodeType.Element, "CommentContent", string.Empty);
                            XmlNode commentContentData = doc.CreateNode(XmlNodeType.CDATA, string.Empty, string.Empty);
                            commentContentData.Value = cm.CommentContent;
                            commentContent.AppendChild(commentContentData);
                            comment.AppendChild(commentContent);
                        }

                        if (cm.AttachmentLinks != null && cm.AttachmentLinks.Count > 0)
                        {
                            XmlNode commentAttachmentLinks = doc.CreateNode(XmlNodeType.Element, "AttachmentLinks", string.Empty);
                            foreach (string link in ann.AttachmentLinks)
                            {
                                XmlNode url = doc.CreateNode(XmlNodeType.Element, "url", string.Empty);
                                url.InnerText = link;
                                commentAttachmentLinks.AppendChild(url);
                            }
                            newSource.AppendChild(commentAttachmentLinks);
                        }
                        comments.AppendChild(comment);
                    }
                    newSource.AppendChild(comments);
                }

                elRoot.AppendChild(newSource);

                XmlTextWriter writer = new XmlTextWriter(Path.Combine(directoryPath, file), Encoding.UTF8);
                writer.Formatting = Formatting.Indented;

                try
                {
                    doc.Save(writer);
                }
                finally
                {
                    writer.Close();
                }
            }
            catch (Exception ex)
            {
                Logger.WriteError("Could not save data of Forum Announcement to XML!", ex);
                throw ex;
            }
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="ann"></param>
        private static void WriteDataClassifiedAd2Xml(Announcement ann)
        {
            try
            {
                string date = DateTime.Now.Year + "-" + DateTime.Now.Month.ToString("00") + "-" + DateTime.Now.Day.ToString("00") + "-" + DateTime.Now.Date.Hour.ToString("00");
                string folder = Utility.GetSourceCodeFolderName(ann.SourceId);
                string directoryPath = Path.Combine(DownloadedFolderPath, ann.SourceCode, folder, date);
                if (!Directory.Exists(directoryPath))
                    Directory.CreateDirectory(directoryPath);

                string file = MiscUtility.ProcessString(UnicodeUtility.UnicodeToNoSign(ann.PressrelaseCate)) + ".xml";
                string fileName = Path.Combine(directoryPath, file);

                // Create the XmlDocument.
                XmlDocument doc = new XmlDocument();
                XmlNode elRoot = null;
                if (File.Exists(fileName))
                {
                    doc.Load(fileName);
                    if (doc.DocumentElement == null)
                    {
                        doc.LoadXml("<posts></posts>");
                    }
                }
                else
                {
                    doc.LoadXml("<posts></posts>");
                }
                elRoot = doc.DocumentElement;

                //Check existed post in xml file.                
                XmlNode Links = doc.SelectSingleNode("./url");
                bool isLinkExisted = false;
                if (Links != null)
                {
                    if (Links.InnerText == ann.Link)
                    {
                        isLinkExisted = true;
                    }
                }
                if (!isLinkExisted)
                {
                    XmlNode newSource = null;
                    newSource = doc.CreateNode(XmlNodeType.Element, "post", string.Empty);

                    if (!string.IsNullOrEmpty(ann.Link))
                    {
                        XmlNode Url = doc.CreateNode(XmlNodeType.Element, "url", string.Empty);
                        Url.InnerText = ann.Link;
                        newSource.AppendChild(Url);
                    }

                    if (ann.DateTime != DateTime.MinValue)
                    {
                        XmlNode Date = doc.CreateNode(XmlNodeType.Element, "date", string.Empty);
                        Date.InnerText = ann.DateTime.ToString();
                        newSource.AppendChild(Date);
                    }

                    if (!string.IsNullOrEmpty(ann.Title))
                    {
                        XmlNode Title = doc.CreateNode(XmlNodeType.Element, "title", string.Empty);
                        Title.InnerText = ann.Title;
                        newSource.AppendChild(Title);
                    }

                    if (!string.IsNullOrEmpty(ann.Message))
                    {
                        XmlNode Message = doc.CreateNode(XmlNodeType.Element, "body", string.Empty);
                        Message.InnerText = ann.Message;
                        newSource.AppendChild(Message);
                    }

                    elRoot.AppendChild(newSource);
                }

                XmlTextWriter writer = new XmlTextWriter(Path.Combine(directoryPath, file), Encoding.UTF8);
                writer.Formatting = Formatting.Indented;
                try
                {
                    doc.Save(writer);
                }
                finally
                {
                    writer.Close();
                }
            }
            catch (Exception ex)
            {
                Logger.WriteError("Could not save data of Chotot Announcement to XML!", ex);
                throw ex;
            }
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="ann"></param>
        private static void WriteDataChotot2Xml(Announcement ann)
        {
            try
            {
                string date = DateTime.Now.Year + "-" + DateTime.Now.Month.ToString("00") + "-" + DateTime.Now.Day.ToString("00") + "-" + DateTime.Now.Date.Hour.ToString("00");
                string folder = Utility.GetSourceCodeFolderName(ann.SourceId);
                string directoryPath = Path.Combine(DownloadedFolderPath, ann.SourceCode, folder, date);
                if (!Directory.Exists(directoryPath))
                    Directory.CreateDirectory(directoryPath);

                string groupFileName = "LoaiKhac";
                string file = groupFileName + ".txt";// MiscUtility.ProcessString(UnicodeUtility.UnicodeToNoSign(ann.PressrelaseCate)) + ".txt";
                string fileName = Path.Combine(directoryPath, file);

                StreamWriter sw = new StreamWriter(Path.Combine(directoryPath, file), true, Encoding.UTF8);
                sw.WriteLine("domain: ");
                sw.WriteLine("category: ");
                sw.WriteLine("needtype: ");
                sw.WriteLine("url: " + ann.Link.ToString());
                sw.WriteLine("time: " + ann.DateTime.ToString());
                sw.WriteLine("title: " + ann.Title);
                sw.WriteLine("body: " + ann.Message.ToString());
                sw.WriteLine("#$#$#$");
                sw.Close();
                /*
                // Create the XmlDocument.
                XmlDocument doc = new XmlDocument();
                XmlNode elRoot = null;
                if (File.Exists(fileName))
                {
                    doc.Load(fileName);
                    if (doc.DocumentElement == null)
                    {
                        doc.LoadXml("<posts></posts>");
                    }
                }
                else
                {
                    doc.LoadXml("<posts></posts>");
                }
                elRoot = doc.DocumentElement;

                //Check existed post in xml file.                
                XmlNode Links = doc.SelectSingleNode("./url");
                bool isLinkExisted = false;
                if (Links != null)
                {
                    if (Links.InnerText == ann.Link)
                    {
                        isLinkExisted = true;
                    }
                }
                if (!isLinkExisted)
                {
                    XmlNode newSource = null;
                    newSource = doc.CreateNode(XmlNodeType.Element, "post", string.Empty);

                    if (!string.IsNullOrEmpty(ann.Link))
                    {
                        XmlNode Url = doc.CreateNode(XmlNodeType.Element, "url", string.Empty);
                        Url.InnerText = ann.Link;
                        newSource.AppendChild(Url);
                    }

                    if (ann.DateTime != DateTime.MinValue)
                    {
                        XmlNode Date = doc.CreateNode(XmlNodeType.Element, "date", string.Empty);
                        Date.InnerText = ann.DateTime.ToString();
                        newSource.AppendChild(Date);
                    }

                    if (!string.IsNullOrEmpty(ann.Title))
                    {
                        XmlNode Title = doc.CreateNode(XmlNodeType.Element, "title", string.Empty);
                        Title.InnerText = ann.Title;
                        newSource.AppendChild(Title);
                    }

                    if (!string.IsNullOrEmpty(ann.Message))
                    {
                        XmlNode Message = doc.CreateNode(XmlNodeType.Element, "body", string.Empty);
                        Message.InnerText = ann.Message;
                        newSource.AppendChild(Message);
                    }

                    elRoot.AppendChild(newSource);
                }

                XmlTextWriter writer = new XmlTextWriter(Path.Combine(directoryPath, file), Encoding.UTF8);
                writer.Formatting = Formatting.Indented;
                try
                {
                    doc.Save(writer);
                }
                finally
                {
                    writer.Close();
                }*/
            }
            catch (Exception ex)
            {
                Logger.WriteError("Could not save data of Chotot Announcement to XML!", ex);
                throw ex;
            }
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="item"></param>
        public static void SaveFailRequests(SourceItem item)
        {
            // Daily Url doesn't need to save failed request to XML
            if (!string.IsNullOrEmpty(item.DailyUrl) && item.Url.Equals(item.DailyUrl, StringComparison.OrdinalIgnoreCase))
            {
                return;
            }

            Monitor.Enter(lockFailRequest);
            try
            {
                rely.Execute(() =>
                {
                    SaveToXml(item);
                });
            }
            finally
            {
                Monitor.Exit(lockFailRequest);
            }
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="item"></param>
        private static void SaveToXml(SourceItem item)
        {
            try
            {
                string sPath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
                string fileName = Path.Combine(sPath, FailRequestsFileName);
                // Create the XmlDocument.
                XmlDocument doc = new XmlDocument();
                XmlNode elRoot = null;
                // Private member to check existed sourceId
                bool isFailIdExisted = false;
                //Check existed fail log file
                if (File.Exists(fileName))
                {
                    FileStream streamReader = new FileStream(fileName, FileMode.Open, FileAccess.Read);
                    doc.Load(streamReader);
                    elRoot = doc.GetElementsByTagName("failrequests").Item(0);
                    streamReader.Close();

                    isFailIdExisted = elRoot.SelectSingleNode("SourceItem[@id='" + item.FailId.ToString() + "']") != null;
                }
                else
                {
                    elRoot = doc.CreateNode(XmlNodeType.Element, "failrequests", string.Empty);
                    doc.AppendChild(elRoot);
                }
                if (!isFailIdExisted)
                {
                    XmlNode newSource = doc.CreateNode(XmlNodeType.Element, "SourceItem", string.Empty);
                    XmlAttribute attrId = doc.CreateAttribute("id");
                    attrId.Value = item.FailId.ToString();
                    newSource.Attributes.Append(attrId);
                    //Save HtmlParserType to fail request log if existed
                    if (!string.IsNullOrEmpty(item.HtmlParserType))
                    {
                        XmlAttribute HtmlParserType = doc.CreateAttribute("HtmlParserType");
                        HtmlParserType.Value = item.HtmlParserType;
                        newSource.Attributes.Append(HtmlParserType);
                    }
                    //Save RequestBuilderType to fail request log if existed
                    if (!string.IsNullOrEmpty(item.RequestBuilderType))
                    {
                        XmlAttribute RequestBuilderType = doc.CreateAttribute("RequestBuilderType");
                        RequestBuilderType.Value = item.RequestBuilderType;
                        newSource.Attributes.Append(RequestBuilderType);
                    }

                    XmlNode lang = doc.CreateNode(XmlNodeType.Element, "LangId", string.Empty);
                    lang.InnerText = item.LangId.ToString();
                    newSource.AppendChild(lang);

                    XmlNode Url = doc.CreateNode(XmlNodeType.Element, "Url", string.Empty);
                    Url.InnerText = item.Url;
                    newSource.AppendChild(Url);

                    XmlNode SourceId = doc.CreateNode(XmlNodeType.Element, "SourceId", string.Empty);
                    SourceId.InnerText = item.SourceId.ToString();
                    newSource.AppendChild(SourceId);

                    XmlNode SourceCode = doc.CreateNode(XmlNodeType.Element, "SourceCode", string.Empty);
                    SourceCode.InnerText = item.SourceCode;
                    newSource.AppendChild(SourceCode);

                    XmlNode SearchCode = doc.CreateNode(XmlNodeType.Element, "SearchCode", string.Empty);
                    SearchCode.InnerText = item.SearchCode;
                    newSource.AppendChild(SearchCode);

                    XmlNode DateTimeFormat = doc.CreateNode(XmlNodeType.Element, "DateTimeFormat", string.Empty);
                    DateTimeFormat.InnerText = item.DateTimeFormat;
                    newSource.AppendChild(DateTimeFormat);

                    XmlNode LastUpdated = doc.CreateNode(XmlNodeType.Element, "LastUpdated", string.Empty);
                    LastUpdated.InnerText = item.LastUpdated.ToString(item.DateTimeFormat);
                    newSource.AppendChild(LastUpdated);

                    XmlNode PrevResponseHtml = doc.CreateNode(XmlNodeType.Element, "PrevResponseHtml", string.Empty);
                    XmlNode viewStateEl = doc.CreateNode(XmlNodeType.CDATA, "", string.Empty);
                    if (!String.IsNullOrEmpty(item.PrevResponseHtml))
                    {
                        string viewState = RegexUtility.ExtractViewState(item.PrevResponseHtml);
                        if (!string.IsNullOrEmpty(viewState))
                            viewStateEl.InnerText = "id=\"__VIEWSTATE\" value=\"" + viewState + "\"";
                    }
                    PrevResponseHtml.AppendChild(viewStateEl);
                    newSource.AppendChild(PrevResponseHtml);

                    elRoot.AppendChild(newSource);

                    XmlTextWriter outStream = new XmlTextWriter(fileName, Encoding.UTF8);
                    outStream.Formatting = Formatting.Indented;
                    doc.Save(outStream);
                    outStream.Close();
                }
            }
            catch (Exception ex)
            {
                Logger.WriteError("Cannot save fail request to XML!", ex);
                throw ex;
            }
        }

        /// <summary>
        /// 
        /// </summary>
        /// <returns></returns>
        public static List<SourceItem> LoadFailRequests()
        {
            List<SourceItem> lstSourceItems = new List<SourceItem>();
            string sPath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
            string fileName = Path.Combine(sPath, FailRequestsFileName);
            try
            {
                if (File.Exists(fileName))
                {
                    XmlDocument doc = new XmlDocument();
                    doc.Load(Path.Combine(sPath, FailRequestsFileName));
                    XmlNodeList SourceItems = doc.GetElementsByTagName("SourceItem");

                    foreach (XmlNode source in SourceItems)
                    {
                        SourceItem item = new SourceItem();

                        item.FailId = long.Parse(source.Attributes["id"].Value);
                        //Read Html Parser Type if existed
                        if (source.Attributes["HtmlParserType"] != null)
                            item.HtmlParserType = source.Attributes["HtmlParserType"].Value;
                        //Read Request Builder Type if existed
                        if (source.Attributes["RequestBuilderType"] != null)
                            item.RequestBuilderType = source.Attributes["RequestBuilderType"].Value;

                        XmlNode lang = source.SelectSingleNode("./LangId");
                        item.LangId = int.Parse(lang.InnerText);

                        XmlNode Url = source.SelectSingleNode("./Url");
                        item.Url = Url.InnerText;

                        XmlNode SourceId = source.SelectSingleNode("./SourceId");
                        item.SourceId = int.Parse(SourceId.InnerText);

                        XmlNode SourceCode = source.SelectSingleNode("./SourceCode");
                        item.SourceCode = SourceCode.InnerText;

                        XmlNode SearchCode = source.SelectSingleNode("./SearchCode");
                        item.SearchCode = SearchCode.InnerText;

                        XmlNode DateTimeFormat = source.SelectSingleNode("./DateTimeFormat");
                        item.DateTimeFormat = DateTimeFormat.InnerText;

                        XmlNode LastUpdated = source.SelectSingleNode("./LastUpdated");
                        if (LastUpdated != null)
                            item.LastUpdated = Utility.ConvertToDateTimeObject(LastUpdated.InnerText, item.DateTimeFormat);

                        XmlNode PrevResponseHtml = source.SelectSingleNode("./PrevResponseHtml");
                        XmlCDataSection cdataSection = PrevResponseHtml.FirstChild as XmlCDataSection;
                        item.PrevResponseHtml = string.IsNullOrEmpty(cdataSection.Value) ? null : cdataSection.Value;

                        lstSourceItems.Add(item);
                    }
                }
            }
            catch (Exception ex)
            {
                Logger.WriteError("Cannot read fail request from XML!", ex);
            }
            return lstSourceItems;
        }

        /// <summary>
        /// 
        /// </summary>
        public static void DeleteFailRequests(long id)
        {
            Monitor.Enter(lockFailRequest);
            try
            {
                rely.Execute(() =>
                {
                    RemoveFailRequest(id);
                });
            }
            finally
            {
                Monitor.Exit(lockFailRequest);
            }

            /*string sPath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
            string XMLFailPath = Path.Combine(sPath, FailRequestsFileName);
            while (true)
            {
                if (!IsFileInUse(XMLFailPath))
                {
                    // take token
                    //isSavingFailRequest = true;
                    // save fail request to file
                    RemoveFailRequest(id);
                    // release token
                    //isSavingFailRequest = false;
                    break;
                }
                else
                {
                    Thread.Sleep(TimeSpan.FromSeconds(1));
                }
            }*/
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="id"></param>
        private static void RemoveFailRequest(long id)
        {
            string sPath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
            string fileName = Path.Combine(sPath, FailRequestsFileName);
            if (File.Exists(fileName))
            {
                try
                {
                    XmlDocument doc = new XmlDocument();
                    doc.Load(fileName);
                    //Get element have id to delete
                    XmlNode sourceId = doc.DocumentElement.SelectSingleNode("SourceItem[@id='" + id.ToString() + "']");
                    if (sourceId != null)
                    {
                        sourceId.ParentNode.RemoveChild(sourceId);
                        StreamWriter outStream = System.IO.File.CreateText(Path.Combine(sPath, FailRequestsFileName));
                        doc.Save(outStream);
                        outStream.Close();
                    }
                }
                catch (Exception ex)
                {
                    Logger.WriteError("Cannot delete fail Request from XML!", ex);
                    throw ex;
                }
            }
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="ann"></param>
        public static void SaveAnnouncementToXml(Announcement ann)
        {

            Monitor.Enter(lockInsertForums);
            try
            {
                rely.Execute(() =>
                {
                    SaveDataToXml(ann);
                });
            }
            finally
            {
                Monitor.Exit(lockInsertForums);
            }
        }

        #endregion

        #region Methods processing Uri/Url/Query

        /// <summary>
        /// 
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static Uri CreateNewUriForNextPage(string url, int hasQuery = 0)
        {
            try
            {
                Uri baseUri = new Uri(url);
                string baseUrl = string.Empty;
                if (baseUri.Segments.Length > 0)
                {
                    switch (hasQuery)
                    {
                        case 0:
                            baseUrl = baseUri.Scheme + "://" + baseUri.Host + baseUri.Segments[0];
                            break;
                        case 1:
                            baseUrl = baseUri.Scheme + "://" + baseUri.Host + baseUri.Segments[0] + baseUri.Segments[1];
                            break;
                        default:
                            baseUrl = baseUri.Scheme + "://" + baseUri.Host + baseUri.Segments[0];
                            break;
                    }
                }
                Uri newUri = new Uri(baseUrl);
                return newUri;
            }
            catch (Exception ex)
            {
                throw ex;
            }
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="url"></param>
        /// <param name="key"></param>
        /// <returns></returns>
        public static string RemoveQueryStringByKey(string url, string key)
        {
            Uri uri = new Uri(url);
            // this gets all the query string key value pairs as a collection
            var newQueryString = HttpUtility.ParseQueryString(uri.Query);
            // this removes the key if exists
            newQueryString.Remove(key);

            // this gets the page path from root without QueryString
            string pagePathWithoutQueryString = uri.GetLeftPart(UriPartial.Path);

            return newQueryString.Count > 0
                ? String.Format("{0}?{1}", pagePathWithoutQueryString, newQueryString)
                : pagePathWithoutQueryString;
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string GetDomainName(string url)
        {
            Uri originalUrl = new Uri(url);
            string domain = originalUrl.Host;
            return domain;
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="s"></param>
        /// <returns></returns>
        public static IDictionary<string, string> ParseQueryString(string s)
        {
            // remove anything other than query string from url
            if (s.Contains("?"))
            {
                s = s.Substring(s.IndexOf('?') + 1);
            }

            var dictionary = new Dictionary<string, string>();

            foreach (string vp in Regex.Split(s, "&"))
            {
                string[] strings = Regex.Split(vp, "=");
                string key = strings[0];
                if (key.Contains("\""))
                {
                    key = key.Replace("\"", string.Empty);
                }
                string value = strings[1];
                if (value.Contains("\""))
                {
                    value = value.Replace("\"", string.Empty);
                }
                dictionary.Add(key, strings.Length == 2 ? HttpUtility.UrlDecode(value) : string.Empty);
            }

            return dictionary;
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="currentPageUrl"></param>
        /// <param name="paramToReplace"></param>
        /// <param name="newValue"></param>
        /// <returns></returns>
        public static string ReplaceQueryStringParameter(string currentPageUrl, string paramToReplace, string newValue)
        {
            var query = ParseQueryString(currentPageUrl);

            query[paramToReplace] = newValue;

            var resultQuery = new StringBuilder();
            bool isFirst = true;

            foreach (KeyValuePair<string, string> pair in query)
            {
                if (!isFirst)
                {
                    resultQuery.Append("&");
                }

                resultQuery.Append(pair.Key);
                resultQuery.Append("=");
                resultQuery.Append(pair.Value);

                isFirst = false;
            }

            var uriBuilder = new UriBuilder(currentPageUrl)
            {
                Query = resultQuery.ToString()
            };

            return uriBuilder.ToString();
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="videoInfo"></param>
        public static void DecryptDownloadUrl(VideoInfo videoInfo)
        {
            IDictionary<string, string> queries = ParseQueryString(videoInfo.DownloadUrl);

            if (queries.ContainsKey(SignatureQuery))
            {
                string encryptedSignature = queries[SignatureQuery];

                string decrypted;

                try
                {
                    decrypted = GetDecipheredSignature(videoInfo.HtmlPlayerVersion, encryptedSignature);
                }

                catch (Exception ex)
                {
                    throw new Exception("Could not decipher signature", ex);
                }

                videoInfo.DownloadUrl = ReplaceQueryStringParameter(videoInfo.DownloadUrl, SignatureQuery, decrypted);
                videoInfo.RequiresDecryption = false;
            }
        }

        /// <summary>
        /// 
        /// </summary>
        /// <param name="htmlPlayerVersion"></param>
        /// <param name="signature"></param>
        /// <returns></returns>
        private static string GetDecipheredSignature(string htmlPlayerVersion, string signature)
        {
            if (signature.Length == CorrectSignatureLength)
            {
                return signature;
            }

            return Decipherer.DecipherWithVersion(signature, htmlPlayerVersion);
        }

        #endregion

        #region Extract forum's comments for otofun and lamchame

        /// <summary>
        /// 
        /// </summary>
        /// <param name="url"></param>
        /// <param name="item"></param>
        /// <returns></returns>
        public static void ExtractComments(string url, SourceItem item, ref List<Comment> comments)
        {
            Logger.Debug("***SCANNING POST COMMENTS URL: " + url);
            HttpStatusCode status = HttpStatusCode.Accepted;
            string html = Utility.LoadHtml(url, ref status);
            Uri newUri = item.SourceId == (int)SourceIds.Lamchame ? Utility.CreateNewUriForNextPage(url, 1) : Utility.CreateNewUriForNextPage(url);
            string nextPageUrl = url;
            try
            {
                if (!string.IsNullOrEmpty(html))
                {
                    string postListId = "id=\"postlist\"";
                    string naviLink = "class=\"navlinks\"";
                    string remainHtml = GetHtmlBody(html, postListId, naviLink);

                    if (!string.IsNullOrEmpty(remainHtml))
                    {
                        Regex regexLiTags = new Regex("<li class=\"postbit postbitim postcontainer\".*?>.*?<hr /></li>");
                        MatchCollection mColLiTags = regexLiTags.Matches(remainHtml);

                        for (int i = 0; i < mColLiTags.Count; i++)
                        {
                            string userName = string.Empty;
                            string commentContent = string.Empty;
                            string dateTime = string.Empty;
                            DateTime postDate = DateTime.Now;

                            string postContent = mColLiTags[i].Value.Trim();
                            postContent = string.Join(string.Empty, Regex.Split(postContent, @"(?:\r\n|\n|\r|\t)"));
                            //Extract date time
                            Regex regexDate = new Regex("<span class=\"date\">.*?</span>");
                            Match mDate = regexDate.Match(postContent);
                            if (mDate.Success)
                            {
                                string dateHtml = mDate.Value.Trim();
                                dateTime = Regex.Replace(RegexUtility.ExtractTextFromHtmlTag(dateHtml), "&nbsp;", " ");
                                if (item.SourceId == (int)SourceIds.Lamchame)
                                {
                                    //Special case for Lamchame forum
                                    //When post comment at today then date post comment was not apppear
                                    //So we can extract time and meger to date of today
                                    Regex regDatePostComment = new Regex(@"\d{2}-\d{2}-\d{4} \d{2}:\d{2}");
                                    Match mDatePost = regDatePostComment.Match(dateTime);
                                    if (mDatePost.Success)
                                    {
                                        dateTime = dateTime.Replace(Regex.Replace(dateTime, @"\d{2}-\d{2}-\d{4} \d{2}:\d{2}", string.Empty).ToString(), string.Empty);
                                    }
                                    else
                                    {
                                        Regex regTime = new Regex("(([0-1][0-9])|([2][0-3])):([0-5][0-9])\\s(?:A|P)M");
                                        Match mTime = regTime.Match(dateTime);
                                        if (mTime.Success)
                                        {
                                            dateTime = DateTime.Now.Date.ToShortDateString() + " " + mTime.Value;
                                        }
                                    }
                                }

                                if (!string.IsNullOrEmpty(dateTime))
                                {
                                    postDate = Utility.ConvertToDateTimeObject(dateTime, item.DateTimeFormat);
                                }
                            }
                            //Extract user name
                            Regex regexUserName = new Regex("<strong>.*?</strong>");
                            Match mUserName = regexUserName.Match(postContent);
                            if (mUserName.Success)
                            {
                                string userNameHtml = mUserName.Value.Trim();
                                userName = RegexUtility.ExtractTextFromHtmlTag(userNameHtml);
                            }
                            //Extract comment content
                            Regex regexContent = new Regex("<blockquote class=\"postcontent restore\">.*?</blockquote></div>");
                            Match mContent = regexContent.Match(postContent);
                            if (mContent.Success)
                            {
                                string contentHtml = mContent.Value.Trim();
                                commentContent = RegexUtility.ExtractTextFromHtmlTag(contentHtml);
                            }
                            if (!string.IsNullOrEmpty(commentContent) && !string.IsNullOrEmpty(userName))
                            {
                                comments.Add(new Comment()
                                {
                                    CommentContent = commentContent,
                                    DateTime = postDate,
                                    UserName = userName,
                                    DateNumberic = long.Parse(
                                        postDate.ToString("yyyyMMddHHmmss")),
                                });
                            }
                        }
                        //Extract url for next page
                        Regex regNextPage = new Regex("<span class=\"selected\">.*?</span>(?<span>.*?</span>)");
                        Match mNextPage = regNextPage.Match(remainHtml);
                        if (mNextPage.Success)
                        {
                            string nextPageHtml = mNextPage.Value;
                            string link = RegexUtility.ExtractHyperlink(nextPageHtml);
                            if (!string.IsNullOrEmpty(link))
                                nextPageUrl = Utility.RemoveQueryStringByKey(new Uri(newUri, link).ToString(), "s");
                        }
                    }
                }
                // if this request is fail request in previous scanning, 
                // remove it when success in current scanning
                if (item.FailId > 0)
                    Utility.DeleteFailRequests(item.FailId);

                Logger.WriteLogInfo("***DONE SCANNING POST COMMENT URL: " + url);
            }
            catch (Exception ex)
            {
                // if has an error, save fail request to next time scanning
                if (item.FailId == 0)
                {
                    item.FailId = DateTime.Now.Ticks;
                    Utility.SaveFailRequests(item);
                }
                // then write log
                string message = string.Format(@"
            Extract post-comment details
            Source code:   {0}
            HTML Parsing:   Fail
            Request URL:    {1}
            Page number:     {2}", item.SourceCode, url, nextPageUrl);
                Logger.WriteError(message, ex);
            }
            // check if has next page 
            // next page counts as a new request, so it needed insert into request queue
            if (CheckNextPage(html) && string.Compare(url, nextPageUrl, false) != 0)
            {
                ExtractComments(nextPageUrl, item, ref comments);
            }
        }

        #endregion

        #region Methods extract html to values

        /// <summary>
        /// 
        /// </summary>
        /// <param name="html"></param>
        /// <param name="startPointHtml"></param>
        /// <param name="endPointHtml"></param>
        /// <returns></returns>
        public static string GetHtmlBody(string html, string startPointHtml, string endPointHtml)
        {
            try
            {
                int indexOfStartHtml = html.IndexOf(startPointHtml);
                int indexOfEndHtml = html.IndexOf(endPointHtml);

                string startHtml = html.Substring(indexOfStartHtml);
                string endHtml = html.Substring(indexOfEndHtml);
                string remainHtml = startHtml.Replace(endHtml, string.Empty);
                remainHtml = string.Join(string.Empty, Regex.Split(remainHtml, @"(?:\r\n|\n|\r|\t)"));
                return remainHtml;
            }
            catch (Exception ex)
            {
                string message = "Fail at GetHtmlBody method:";
                Logger.WriteError(message, ex);
                Logger.WriteLogInfo("**************FAIL EXTRACT HTML BODY**************");
                return string.Empty;
            }
        }

        #endregion
    }
}
